Parameters Comparison

rm(list = ls())
source("function.R")

1 Fit and Test with CV (run once)

# InD_Dataset = "MNIST"
# OOD_Datasets = c("FashionMNIST", "Cifar_10", "SVHN", "Imagenet_r", "Imagenet_c")
# model_fit_test(trainset = InD_Dataset, testsets = OOD_Datasets)  # Run only once

# InD_Dataset = "FashionMNIST"
# OOD_Datasets = c("MNIST", "Cifar_10", "SVHN", "Imagenet_r", "Imagenet_c")
# model_fit_test(trainset = InD_Dataset, testsets = OOD_Datasets)  # Run only once

1.1 MNIST

InD_Dataset = "MNIST"
OOD_Datasets = c("FashionMNIST", "Cifar_10", "SVHN", "Imagenet_r", "Imagenet_c")

1.1.0.1 KL distance

result = score_function(InD_Dataset, OOD_Datasets[1])
hist_plot_KL(result$test.df, result$ood.df, OOD_Datasets[1])

result = score_function(InD_Dataset, OOD_Datasets[2])
hist_plot_KL(result$test.df, result$ood.df, OOD_Datasets[2])

result = score_function(InD_Dataset, OOD_Datasets[3])
hist_plot_KL(result$test.df, result$ood.df, OOD_Datasets[3])

result = score_function(InD_Dataset, OOD_Datasets[4])
hist_plot_KL(result$test.df, result$ood.df, OOD_Datasets[4])

result = score_function(InD_Dataset, OOD_Datasets[5])
hist_plot_KL(result$test.df, result$ood.df, OOD_Datasets[5])

1.1.0.2 Mean

result = score_function(InD_Dataset, OOD_Datasets[1])
hist_plot_meanSD(result$test.df, result$ood.df, result$cv_results, OOD_Datasets[1], "mean")

result = score_function(InD_Dataset, OOD_Datasets[2])
hist_plot_meanSD(result$test.df, result$ood.df, result$cv_results, OOD_Datasets[2], "mean")

result = score_function(InD_Dataset, OOD_Datasets[3])
hist_plot_meanSD(result$test.df, result$ood.df, result$cv_results, OOD_Datasets[3], "mean")

result = score_function(InD_Dataset, OOD_Datasets[4])
hist_plot_meanSD(result$test.df, result$ood.df, result$cv_results, OOD_Datasets[4], "mean")

result = score_function(InD_Dataset, OOD_Datasets[5])
hist_plot_meanSD(result$test.df, result$ood.df, result$cv_results, OOD_Datasets[5], "mean")

1.1.0.3 SD

result = score_function(InD_Dataset, OOD_Datasets[1])
hist_plot_meanSD(result$test.df, result$ood.df, result$cv_results, OOD_Datasets[1], "std")

result = score_function(InD_Dataset, OOD_Datasets[2])
hist_plot_meanSD(result$test.df, result$ood.df, result$cv_results, OOD_Datasets[2], "std")

result = score_function(InD_Dataset, OOD_Datasets[3])
hist_plot_meanSD(result$test.df, result$ood.df, result$cv_results, OOD_Datasets[3], "std")

result = score_function(InD_Dataset, OOD_Datasets[4])
hist_plot_meanSD(result$test.df, result$ood.df, result$cv_results, OOD_Datasets[4], "std")

result = score_function(InD_Dataset, OOD_Datasets[5])
hist_plot_meanSD(result$test.df, result$ood.df, result$cv_results, OOD_Datasets[5], "std")

1.2 FashionMNIST

InD_Dataset = "FashionMNIST"
OOD_Datasets = c("MNIST", "Cifar_10", "SVHN", "Imagenet_r", "Imagenet_c")

1.2.0.1 KL distance

result = score_function(InD_Dataset, OOD_Datasets[1])
hist_plot_KL(result$test.df, result$ood.df, OOD_Datasets[1])

result = score_function(InD_Dataset, OOD_Datasets[2])
hist_plot_KL(result$test.df, result$ood.df, OOD_Datasets[2])

result = score_function(InD_Dataset, OOD_Datasets[3])
hist_plot_KL(result$test.df, result$ood.df, OOD_Datasets[3])

result = score_function(InD_Dataset, OOD_Datasets[4])
hist_plot_KL(result$test.df, result$ood.df, OOD_Datasets[4])

result = score_function(InD_Dataset, OOD_Datasets[5])
hist_plot_KL(result$test.df, result$ood.df, OOD_Datasets[5])

1.2.0.2 Mean

result = score_function(InD_Dataset, OOD_Datasets[1])
hist_plot_meanSD(result$test.df, result$ood.df, result$cv_results, OOD_Datasets[1], "mean")

result = score_function(InD_Dataset, OOD_Datasets[2])
hist_plot_meanSD(result$test.df, result$ood.df, result$cv_results, OOD_Datasets[2], "mean")

result = score_function(InD_Dataset, OOD_Datasets[3])
hist_plot_meanSD(result$test.df, result$ood.df, result$cv_results, OOD_Datasets[3], "mean")

result = score_function(InD_Dataset, OOD_Datasets[4])
hist_plot_meanSD(result$test.df, result$ood.df, result$cv_results, OOD_Datasets[4], "mean")

result = score_function(InD_Dataset, OOD_Datasets[5])
hist_plot_meanSD(result$test.df, result$ood.df, result$cv_results, OOD_Datasets[5], "mean")

1.2.0.3 SD

result = score_function(InD_Dataset, OOD_Datasets[1])
hist_plot_meanSD(result$test.df, result$ood.df, result$cv_results, OOD_Datasets[1], "std")

result = score_function(InD_Dataset, OOD_Datasets[2])
hist_plot_meanSD(result$test.df, result$ood.df, result$cv_results, OOD_Datasets[2], "std")

result = score_function(InD_Dataset, OOD_Datasets[3])
hist_plot_meanSD(result$test.df, result$ood.df, result$cv_results, OOD_Datasets[3], "std")

result = score_function(InD_Dataset, OOD_Datasets[4])
hist_plot_meanSD(result$test.df, result$ood.df, result$cv_results, OOD_Datasets[4], "std")

result = score_function(InD_Dataset, OOD_Datasets[5])
hist_plot_meanSD(result$test.df, result$ood.df, result$cv_results, OOD_Datasets[5], "std")